#!/usr/bin/env bash

source ./config/env.sh
clear

mkdir -p ${K8S_CA_DIR}/flannel

cat > ${K8S_CA_DIR}/flannel/flanneld-csr.json <<EOF
{
  "CN": "flanneld",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF

cd ${K8S_CA_DIR}/flannel

cfssl gencert -ca=${K8S_CA_DIR}/admin/ca.pem \
  -ca-key=${K8S_CA_DIR}/admin/ca-key.pem \
  -config=${K8S_CA_DIR}/admin/ca-config.json \
  -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld

for master_ip in ${MASTER_IP}
do
    echo "=========================================== ${master_ip} ==========================================="
    ssh root@${master_ip} "test -e ${K8S_CA_DIR}/flannel || mkdir -p ${K8S_CA_DIR}/flannel"
    scp ${K8S_CA_DIR}/flannel/flanneld*.pem root@${master_ip}:${K8S_CA_DIR}/flannel
done

${K8S_BIN_DIR}/etcdctl \
  --endpoints=${ETCD_NODES} \
  --ca-file=${K8S_CA_DIR}/admin/ca.pem \
  --cert-file=${K8S_CA_DIR}/flannel/flanneld.pem \
  --key-file=${K8S_CA_DIR}/flannel/flanneld-key.pem \
  set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 21, "Backend": {"Type": "vxlan"}}'

mkdir -p ${K8S_DATA}/config/flannel

cat > ${K8S_DATA}/config/flannel/flanneld.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service

[Service]
Type=notify
ExecStart=${K8S_BIN_DIR}/flanneld \\
  -etcd-cafile=${K8S_CA_DIR}/admin/ca.pem \\
  -etcd-certfile=${K8S_CA_DIR}/flannel/flanneld.pem \\
  -etcd-keyfile=${K8S_CA_DIR}/flannel/flanneld-key.pem \\
  -etcd-endpoints=${ETCD_NODES} \\
  -etcd-prefix=${FLANNEL_ETCD_PREFIX} \\
  -iface=${FLANNEL_INTERFACE} \\
  -ip-masq
ExecStartPost=${K8S_BIN_DIR}/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=always
RestartSec=5
StartLimitInterval=0

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

for master_ip in ${MASTER_IP}
do
  scp ${K8S_DATA}/config/flannel/flanneld.service root@${master_ip}:/etc/systemd/system/
done



for master_ip in ${MASTER_IP}
do
  echo "=========================================== ${master_ip} ==========================================="
  ssh root@${master_ip} "systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld"
done

for master_ip in ${MASTER_IP}
do
  echo "=========================================== ${master_ip} ==========================================="
  ssh root@${master_ip} "systemctl restart flanneld"
done

# 查看状态
for master_ip in ${MASTER_IP}
do
  echo "=========================================== ${master_ip} ==========================================="
  ssh root@${master_ip} "sudo systemctl status flanneld | grep Active"
done


# 查看集群 Pod 网段(/16）
sleep 5
${K8S_BIN_DIR}/etcdctl \
  --endpoints=${ETCD_NODES} \
  --ca-file=${K8S_CA_DIR}/admin/ca.pem \
  --cert-file=${K8S_CA_DIR}/flannel/flanneld.pem \
  --key-file=${K8S_CA_DIR}/flannel/flanneld-key.pem \
  get ${FLANNEL_ETCD_PREFIX}/config >> ${K8S_LOGS_DIR}/flanneld-cidr-msg.log

# 查看已分配的 Pod 子网段列表(/24)
sleep 5
${K8S_BIN_DIR}/etcdctl \
  --endpoints=${ETCD_NODES} \
  --ca-file=${K8S_CA_DIR}/admin/ca.pem \
  --cert-file=${K8S_CA_DIR}/flannel/flanneld.pem \
  --key-file=${K8S_CA_DIR}/flannel/flanneld-key.pem \
  ls ${FLANNEL_ETCD_PREFIX}/subnets >> ${K8S_LOGS_DIR}/flanneld-cidr-msg.log

# 查看某一 Pod 网段对应的节点 IP 和 flannel 接口地址:
#etcdctl \
#  --endpoints=${ETCD_ENDPOINTS} \
#  --ca-file=/etc/kubernetes/cert/ca.pem \
#  --cert-file=/etc/flanneld/cert/flanneld.pem \
#  --key-file=/etc/flanneld/cert/flanneld-key.pem \
#  get ${FLANNEL_ETCD_PREFIX}/subnets/172.30.96.0-21


# 验证各节点能通过 Pod 网段互通
for master_ip in ${MASTER_IP}
do
  echo ">>>>>>>>>>>>>>>>>>>>>>${master_ip}<<<<<<<<<<<<<<<<<<<<<"
  ssh root@${master_ip} "/usr/sbin/ip addr show flannel.1|grep -w inet"
done


if [ $? == 0 ]
then
  echo "write flanneld in etcd succeed ${K8S_LOGS_DIR}/flanneld-cidr-msg.log"
else
  echo -e "\033[31m write flanneld in etcd succeed failed\033[0m" 
  exit
fi